R Notebook

library(rtweet)
library(ggplot2)
library(tidyr)
library(dplyr)
library(tidytext)
library(stringr)
library(scales)
library(readr)
library(lubridate)
library(vader)
library(topicmodels)
library(quanteda)
library(lubridate)
library(zoo)
library(plotly)
library(forcats)
library(igraph)
library(ggraph)
library(widyr)
library(qdapRegex)
library(magick)

library(RedditExtractoR)
reddit_posts <- RedditExtractoR::reddit_urls(search_terms = "Covid", subreddit = "southafrica", page_threshold = 10)

#top words
top_words_reddit <- reddit_media_df %>%
  anti_join(stop_words) %>%
  count(word) %>%
  arrange(desc(n))

Joining, by = "word"

top_words_reddit %>%
  slice(1:20) %>%
  ggplot(aes(reorder(word,-n), n, fill = word)) +
  geom_bar(stat = "identity") +
  theme_minimal() +
  theme(
    axis.text.x = element_text(
      angle = 60,
      hjust = 1,
      size = 13
    ),
    plot.title = element_text(hjust = 0.5, size = 18)
  ) +
  ylab("Frequency") +
  xlab ("") +
  ggtitle("Most frequent reddit posts") +
  guides(fill = FALSE)

#bigrams
reddit_bigram_df <- reddit_posts_df %>%
  #filter(str_detect(text, fixed(covid_dictionary, ignore_case = TRUE))) %>%
  filter(!str_detect(title, "^RT")) %>%
  mutate(title = gsub(" ?(f|ht)tp(s?)://(.*)[.][a-z]+", " ", title)) %>%
  unnest_tokens(bigram, title, token = "ngrams", n = 2) %>%
  separate(bigram, c("word1", "word2"), sep = " ")

reddit_bigrams_filtered <- reddit_bigram_df %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word)

# new bigram counts:
reddit_bigram_counts <- reddit_bigrams_filtered %>%
  count(word1, word2, sort = TRUE)

reddit_bigrams_united <- reddit_bigrams_filtered %>%
  unite(bigram, word1, word2, sep = " ")

#weights and graphs
reddit_bigram_graph <- reddit_bigram_counts %>%
  filter(n > 100) %>%
  graph_from_data_frame()

reddit_section_words <- reddit_media_df %>%
  mutate(section = row_number() %/% 10) %>%
  filter(section > 0)

# count words co-occuring within sections
reddit_word_pairs <- reddit_section_words %>%
  pairwise_count(word, section, sort = TRUE)

# we need to filter for at least relatively common words first
reddit_word_cors <- reddit_section_words %>%
  group_by(word) %>%
  filter(n() >= 20) %>%
  pairwise_cor(word, section, sort = TRUE)

reddit_word_cors %>%
  slice_max(correlation, n = 6) %>%
  mutate(item2 = reorder(item2, correlation)) %>%
  ggplot(aes(item2, correlation)) +
  geom_bar(stat = "identity") +
  facet_wrap( ~ item1, scales = "free") +
  coord_flip()

set.seed(1234)
reddit_word_cors %>%
  graph_from_data_frame() %>%
  ggraph(layout = "fr") +
  geom_edge_link(aes(edge_alpha = correlation),
                 show.legend = FALSE,
                 edge_width = 3) +
  geom_node_point(color = "lightblue", size = 5) +
  geom_node_text(aes(label = name), repel = TRUE) +
  theme_void()

# vader_df <- vader_df(reddit_posts_df$title)
# write_as_csv(vader_df, "data_in/vader_reddit_posts")

vader_df <- read_csv("data_in/vader_reddit_posts.csv")

-- Column specification --------------------------------------------------------
cols(
  text = col_character(),
  word_scores = col_character(),
  compound = col_double(),
  pos = col_double(),
  neu = col_double(),
  neg = col_double(),
  but_count = col_double()
)

vader_df <- vader_df %>% mutate("X1" = row_number())
reddit_posts_df <- reddit_posts_df %>% mutate("X1" = row_number())
reddit_vader_df <- reddit_posts_df %>% left_join(vader_df, by = "X1")

#General sentiment over time. Media agencies seem to have an even number of positive and negative tweets per day.
ggplot(
  reddit_vader_df %>%
    mutate(date = as.Date(date)) %>%
    group_by(date) %>%
    mutate(compound = replace(compound, is.na(compound), 0)) %>%
    summarise(sum_compound = sum(as.double(compound))),
  aes(date, sum_compound)
) +
  geom_bar(stat = "identity") +
  geom_smooth(method = "lm", na.rm = TRUE, se = FALSE) +
  theme_minimal() +
  scale_y_continuous(expand = c(0, 0)) +
  scale_x_date(date_breaks = "3 day", date_labels = "%m-%d") +
  theme(
    axis.text.x = element_text(
      angle = 90,
      vjust = 0.5,
      hjust = 1
    ),
    panel.grid.minor.y = element_blank(),
    panel.grid.minor.x = element_blank(),
  )

# An illegal gold miner who was severely injured in a clash in which four other illegal miners were killed has been charged for their murders Mpumalanga police said on Thursday
# Italy midfielder Jorginho said team glory is more important than any push to win a surprise Ballon d’Or award for the best player in world football after a teammate said he deserved it

ggplot(
  reddit_vader_df %>%
    mutate(date = as.Date(date)) %>%
    group_by(date) %>%
    summarise(compound),
  aes(date, compound)
) +
  geom_bar(stat = "identity") +
  geom_smooth(method = "lm", na.rm = TRUE) +
  theme_minimal() +
  scale_y_continuous(expand = c(0, 0))

`summarise()` has grouped output by 'date'. You can override using the `.groups` argument.

#News sentiment
reddit_vader_df %>%
  mutate(date = as.Date(date)) %>%
  group_by(date) %>%
  summarise(compound) %>%
  arrange(desc(compound))

`summarise()` has grouped output by 'date'. You can override using the `.groups` argument.

ggplot(reddit_vader_df %>% arrange(compound) %>% mutate(X1 = factor(X1, levels = X1)),
       aes(x = X1)) +
  geom_point(mapping =  aes(y = compound)) +
  theme_minimal() +
  scale_y_continuous(expand = c(0, 0))

reddit_vader_df %>%
  mutate(date = as.Date(date)) %>%
  group_by(date) %>%
  mutate(compound = replace(compound, is.na(compound), 0)) %>%
  summarise(sum_compound = sum(as.double(compound)))

reddit_matrix <-
  reddit_media_df %>% sample(size = 0.1) %>%  count(subreddit, word) %>% cast_dfm(subreddit, word, n)

Error: Must group by variables found in `.data`.
* Column `subreddit` is not found.
* Column `word` is not found.
Run `rlang::last_error()` to see where the error occurred.

library("ldatuning")
library("topicmodels")

data("AssociatedPress", package = "topicmodels")
dtm <- AssociatedPress[1:10,]

result <- FindTopicsNumber(
  reddit_matrix,
  topics = seq(from = 2, to = 15, by = 1),
  metrics = c("Griffiths2004", "CaoJuan2009", "Arun2010", "Deveaud2014"),
  method = "Gibbs",
  control = list(seed = 77),
  mc.cores = 2L,
  verbose = TRUE
)

FindTopicsNumber_plot(result)